

<!DOCTYPE html>
<html class="writer-html5" lang="zh" >
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>蜘蛛中间件 &mdash; Scrapy 2.3.0 文档</title>
  

  
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster.custom.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster.bundle.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-shadow.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-punk.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-noir.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-light.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-borderless.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/micromodal.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/sphinx_rtd_theme.css" type="text/css" />

  
  
  
  

  
  <!--[if lt IE 9]>
    <script src="../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script src="../_static/jquery.js"></script>
        <script src="../_static/underscore.js"></script>
        <script src="../_static/doctools.js"></script>
        <script src="../_static/language_data.js"></script>
        <script src="../_static/js/hoverxref.js"></script>
        <script src="../_static/js/tooltipster.bundle.min.js"></script>
        <script src="../_static/js/micromodal.min.js"></script>
    
    <script type="text/javascript" src="../_static/js/theme.js"></script>

    
    <link rel="index" title="索引" href="../genindex.html" />
    <link rel="search" title="搜索" href="../search.html" />
    <link rel="next" title="扩展" href="extensions.html" />
    <link rel="prev" title="下载器中间件" href="downloader-middleware.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../index.html" class="icon icon-home" alt="Documentation Home"> Scrapy
          

          
          </a>

          
            
            
              <div class="version">
                2.3
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">第一步</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../intro/overview.html">Scrapy一目了然</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/install.html">安装指南</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/tutorial.html">Scrapy 教程</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/examples.html">实例</a></li>
</ul>
<p class="caption"><span class="caption-text">基本概念</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="commands.html">命令行工具</a></li>
<li class="toctree-l1"><a class="reference internal" href="spiders.html">蜘蛛</a></li>
<li class="toctree-l1"><a class="reference internal" href="selectors.html">选择器</a></li>
<li class="toctree-l1"><a class="reference internal" href="items.html">项目</a></li>
<li class="toctree-l1"><a class="reference internal" href="loaders.html">项目加载器</a></li>
<li class="toctree-l1"><a class="reference internal" href="shell.html">Scrapy shell</a></li>
<li class="toctree-l1"><a class="reference internal" href="item-pipeline.html">项目管道</a></li>
<li class="toctree-l1"><a class="reference internal" href="feed-exports.html">Feed 导出</a></li>
<li class="toctree-l1"><a class="reference internal" href="request-response.html">请求和响应</a></li>
<li class="toctree-l1"><a class="reference internal" href="link-extractors.html">链接提取器</a></li>
<li class="toctree-l1"><a class="reference internal" href="settings.html">设置</a></li>
<li class="toctree-l1"><a class="reference internal" href="exceptions.html">例外情况</a></li>
</ul>
<p class="caption"><span class="caption-text">内置服务</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="logging.html">登录</a></li>
<li class="toctree-l1"><a class="reference internal" href="stats.html">统计数据集合</a></li>
<li class="toctree-l1"><a class="reference internal" href="email.html">发送电子邮件</a></li>
<li class="toctree-l1"><a class="reference internal" href="telnetconsole.html">远程登录控制台</a></li>
<li class="toctree-l1"><a class="reference internal" href="webservice.html">Web服务</a></li>
</ul>
<p class="caption"><span class="caption-text">解决具体问题</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../faq.html">常见问题</a></li>
<li class="toctree-l1"><a class="reference internal" href="debug.html">调试spiders</a></li>
<li class="toctree-l1"><a class="reference internal" href="contracts.html">蜘蛛合约</a></li>
<li class="toctree-l1"><a class="reference internal" href="practices.html">常用做法</a></li>
<li class="toctree-l1"><a class="reference internal" href="broad-crawls.html">宽爬行</a></li>
<li class="toctree-l1"><a class="reference internal" href="developer-tools.html">使用浏览器的开发人员工具进行抓取</a></li>
<li class="toctree-l1"><a class="reference internal" href="dynamic-content.html">选择动态加载的内容</a></li>
<li class="toctree-l1"><a class="reference internal" href="leaks.html">调试内存泄漏</a></li>
<li class="toctree-l1"><a class="reference internal" href="media-pipeline.html">下载和处理文件和图像</a></li>
<li class="toctree-l1"><a class="reference internal" href="deploy.html">部署蜘蛛</a></li>
<li class="toctree-l1"><a class="reference internal" href="autothrottle.html">AutoThrottle 扩展</a></li>
<li class="toctree-l1"><a class="reference internal" href="benchmarking.html">标杆管理</a></li>
<li class="toctree-l1"><a class="reference internal" href="jobs.html">作业：暂停和恢复爬行</a></li>
<li class="toctree-l1"><a class="reference internal" href="coroutines.html">协同程序</a></li>
<li class="toctree-l1"><a class="reference internal" href="asyncio.html">asyncio</a></li>
</ul>
<p class="caption"><span class="caption-text">扩展Scrapy</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="architecture.html">体系结构概述</a></li>
<li class="toctree-l1"><a class="reference internal" href="downloader-middleware.html">下载器中间件</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">蜘蛛中间件</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#activating-a-spider-middleware">激活蜘蛛中间件</a></li>
<li class="toctree-l2"><a class="reference internal" href="#writing-your-own-spider-middleware">编写自己的蜘蛛中间件</a></li>
<li class="toctree-l2"><a class="reference internal" href="#built-in-spider-middleware-reference">内置蜘蛛中间件参考</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#module-scrapy.spidermiddlewares.depth">DepthMiddleware</a></li>
<li class="toctree-l3"><a class="reference internal" href="#module-scrapy.spidermiddlewares.httperror">HttpErrorMiddleware</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#httperrormiddleware-settings">httpErrorMiddleware设置</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="#module-scrapy.spidermiddlewares.offsite">OffsiteMiddleware</a></li>
<li class="toctree-l3"><a class="reference internal" href="#module-scrapy.spidermiddlewares.referer">RefererMiddleware</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#referermiddleware-settings">ReferermIddleware设置</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="#module-scrapy.spidermiddlewares.urllength">UrlLengthMiddleware</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="extensions.html">扩展</a></li>
<li class="toctree-l1"><a class="reference internal" href="api.html">核心API</a></li>
<li class="toctree-l1"><a class="reference internal" href="signals.html">信号</a></li>
<li class="toctree-l1"><a class="reference internal" href="exporters.html">条目导出器</a></li>
</ul>
<p class="caption"><span class="caption-text">其余所有</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../news.html">发行说明</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contributing.html">为 Scrapy 贡献</a></li>
<li class="toctree-l1"><a class="reference internal" href="../versioning.html">版本控制和API稳定性</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">Scrapy</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
        
      <li>蜘蛛中间件</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="spider-middleware">
<span id="topics-spider-middleware"></span><h1>蜘蛛中间件<a class="headerlink" href="#spider-middleware" title="永久链接至标题">¶</a></h1>
<p>Spider中间件是一个钩子框架，可以钩住Scrapy的Spider处理机制，在该机制中，您可以插入自定义功能来处理发送到的响应。 <a class="reference internal" href="spiders.html#topics-spiders"><span class="std std-ref">蜘蛛</span></a> 用于处理和处理由spider生成的请求和项目。</p>
<div class="section" id="activating-a-spider-middleware">
<span id="topics-spider-middleware-setting"></span><h2>激活蜘蛛中间件<a class="headerlink" href="#activating-a-spider-middleware" title="永久链接至标题">¶</a></h2>
<p>要激活蜘蛛中间件组件，请将其添加到 <a class="reference internal" href="settings.html#std-setting-SPIDER_MIDDLEWARES"><code class="xref std std-setting docutils literal notranslate"><span class="pre">SPIDER_MIDDLEWARES</span></code></a> 设置，这是一个dict，其键是中间件类路径，其值是中间件命令。</p><script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script>
<ins class="adsbygoogle"
     style="display:block; text-align:center;"
     data-ad-layout="in-article"
     data-ad-format="fluid"
     data-ad-client="ca-pub-1466963416408457"
     data-ad-slot="8850786025"></ins>
<script>
     (adsbygoogle = window.adsbygoogle || []).push({});
</script>
<p>举个例子：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">SPIDER_MIDDLEWARES</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s1">&#39;myproject.middlewares.CustomSpiderMiddleware&#39;</span><span class="p">:</span> <span class="mi">543</span><span class="p">,</span>
<span class="p">}</span>
</pre></div>
</div>
<p>这个 <a class="reference internal" href="settings.html#std-setting-SPIDER_MIDDLEWARES"><code class="xref std std-setting docutils literal notranslate"><span class="pre">SPIDER_MIDDLEWARES</span></code></a> 设置与合并 <a class="reference internal" href="settings.html#std-setting-SPIDER_MIDDLEWARES_BASE"><code class="xref std std-setting docutils literal notranslate"><span class="pre">SPIDER_MIDDLEWARES_BASE</span></code></a> 在scrappy中定义的设置（不打算被重写），然后按顺序排序，以获得已启用中间件的最终排序列表：第一个中间件更接近引擎，最后一个更接近spider。也就是说， <a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_input" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_input"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_input()</span></code></a> 每个中间件的方法将以增加的中间件顺序（100、200、300，…）调用，并且 <a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_output()</span></code></a> 每个中间件的方法将按降序调用。</p>
<p>要决定分配给中间件的顺序，请参见 <a class="reference internal" href="settings.html#std-setting-SPIDER_MIDDLEWARES_BASE"><code class="xref std std-setting docutils literal notranslate"><span class="pre">SPIDER_MIDDLEWARES_BASE</span></code></a> 根据要插入中间件的位置设置和选择一个值。顺序很重要，因为每个中间件执行不同的操作，并且您的中间件可能依赖于之前（或之后）应用的一些中间件。</p>
<p>如果要禁用内置中间件（定义于 <a class="reference internal" href="settings.html#std-setting-SPIDER_MIDDLEWARES_BASE"><code class="xref std std-setting docutils literal notranslate"><span class="pre">SPIDER_MIDDLEWARES_BASE</span></code></a> ，默认情况下启用）您必须在项目中定义它。 <a class="reference internal" href="settings.html#std-setting-SPIDER_MIDDLEWARES"><code class="xref std std-setting docutils literal notranslate"><span class="pre">SPIDER_MIDDLEWARES</span></code></a> 设置和分配 <code class="docutils literal notranslate"><span class="pre">None</span></code> 作为其价值。例如，如果要禁用非现场中间件：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">SPIDER_MIDDLEWARES</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s1">&#39;myproject.middlewares.CustomSpiderMiddleware&#39;</span><span class="p">:</span> <span class="mi">543</span><span class="p">,</span>
    <span class="s1">&#39;scrapy.spidermiddlewares.offsite.OffsiteMiddleware&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="p">}</span>
</pre></div>
</div>
<p>最后，请记住，某些中间商可能需要通过特定设置启用。有关更多信息，请参阅每个中间件文档。</p>
</div>
<div class="section" id="writing-your-own-spider-middleware">
<span id="custom-spider-middleware"></span><h2>编写自己的蜘蛛中间件<a class="headerlink" href="#writing-your-own-spider-middleware" title="永久链接至标题">¶</a></h2>
<p>每个蜘蛛中间件都是一个python类，它定义了下面定义的一个或多个方法。</p>
<p>主要入口点是 <code class="docutils literal notranslate"><span class="pre">from_crawler</span></code> 类方法，它接收 <a class="reference internal" href="api.html#scrapy.crawler.Crawler" title="scrapy.crawler.Crawler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Crawler</span></code></a> 实例。这个 <a class="reference internal" href="api.html#scrapy.crawler.Crawler" title="scrapy.crawler.Crawler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Crawler</span></code></a> 例如，对象允许您访问 <a class="reference internal" href="settings.html#topics-settings"><span class="std std-ref">settings</span></a> .</p>
<span class="target" id="module-scrapy.spidermiddlewares"></span><dl class="py class">
<dt id="scrapy.spidermiddlewares.SpiderMiddleware">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.</code><code class="sig-name descname">SpiderMiddleware</code><a class="headerlink" href="#scrapy.spidermiddlewares.SpiderMiddleware" title="永久链接至目标">¶</a></dt>
<dd><dl class="py method">
<dt id="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_input">
<code class="sig-name descname">process_spider_input</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">response</span></em>, <em class="sig-param"><span class="n">spider</span></em><span class="sig-paren">)</span><a class="headerlink" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_input" title="永久链接至目标">¶</a></dt>
<dd><p>对于通过蜘蛛中间件进入蜘蛛进行处理的每个响应，都会调用此方法。</p>
<p><a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_input" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_input"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_input()</span></code></a> 应该返回 <code class="docutils literal notranslate"><span class="pre">None</span></code> 或者提出例外。</p>
<p>如果它回来 <code class="docutils literal notranslate"><span class="pre">None</span></code> ，scrapy将继续处理此响应，执行所有其他中间软件，直到最后将响应提交给spider进行处理。</p>
<p>如果它引发了一个异常，那么Scrapy就不会麻烦调用任何其他的蜘蛛中间件了。 <a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_input" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_input"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_input()</span></code></a> 如果有请求，将调用errback，否则将启动 <a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_exception()</span></code></a> 链条。errback的输出被链接回另一个方向 <a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_output()</span></code></a> 处理，或 <a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_exception()</span></code></a> 如果它引发了一个异常。</p>
<dl class="field-list simple">
<dt class="field-odd">参数</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>response</strong> (<a class="reference internal" href="request-response.html#scrapy.http.Response" title="scrapy.http.Response"><code class="xref py py-class docutils literal notranslate"><span class="pre">Response</span></code></a> object) -- 正在处理的响应</p></li>
<li><p><strong>spider</strong> (<a class="reference internal" href="spiders.html#scrapy.spiders.Spider" title="scrapy.spiders.Spider"><code class="xref py py-class docutils literal notranslate"><span class="pre">Spider</span></code></a> object) -- 此响应所针对的蜘蛛</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="py method">
<dt id="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output">
<code class="sig-name descname">process_spider_output</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">response</span></em>, <em class="sig-param"><span class="n">result</span></em>, <em class="sig-param"><span class="n">spider</span></em><span class="sig-paren">)</span><a class="headerlink" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output" title="永久链接至目标">¶</a></dt>
<dd><p>在处理完响应后，调用此方法，并返回蜘蛛返回的结果。</p>
<p><a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_output()</span></code></a> 必须返回 <a class="reference internal" href="request-response.html#scrapy.http.Request" title="scrapy.http.Request"><code class="xref py py-class docutils literal notranslate"><span class="pre">Request</span></code></a> 对象和 <a class="reference internal" href="items.html#topics-items"><span class="std std-ref">item object</span></a> .</p>
<dl class="field-list simple">
<dt class="field-odd">参数</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>response</strong> (<a class="reference internal" href="request-response.html#scrapy.http.Response" title="scrapy.http.Response"><code class="xref py py-class docutils literal notranslate"><span class="pre">Response</span></code></a> object) -- 从spider生成此输出的响应</p></li>
<li><p><strong>result</strong> (an iterable of <a class="reference internal" href="request-response.html#scrapy.http.Request" title="scrapy.http.Request"><code class="xref py py-class docutils literal notranslate"><span class="pre">Request</span></code></a> objects and
<a class="reference internal" href="items.html#topics-items"><span class="std std-ref">item object</span></a>) -- 蜘蛛返回的结果</p></li>
<li><p><strong>spider</strong> (<a class="reference internal" href="spiders.html#scrapy.spiders.Spider" title="scrapy.spiders.Spider"><code class="xref py py-class docutils literal notranslate"><span class="pre">Spider</span></code></a> object) -- 正在处理结果的蜘蛛</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="py method">
<dt id="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception">
<code class="sig-name descname">process_spider_exception</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">response</span></em>, <em class="sig-param"><span class="n">exception</span></em>, <em class="sig-param"><span class="n">spider</span></em><span class="sig-paren">)</span><a class="headerlink" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception" title="永久链接至目标">¶</a></dt>
<dd><p>当蜘蛛或 <a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_output()</span></code></a> 方法（来自以前的蜘蛛中间件）引发异常。</p>
<p><a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_exception()</span></code></a> 应该返回 <code class="docutils literal notranslate"><span class="pre">None</span></code> 或者是 <a class="reference internal" href="request-response.html#scrapy.http.Request" title="scrapy.http.Request"><code class="xref py py-class docutils literal notranslate"><span class="pre">Request</span></code></a> 对象和 <a class="reference internal" href="items.html#topics-items"><span class="std std-ref">item object</span></a> .</p>
<p>如果它回来 <code class="docutils literal notranslate"><span class="pre">None</span></code> ，Scrapy将继续处理此异常，执行任何其他 <a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_exception()</span></code></a> 在下面的中间件组件中，直到没有中间件组件离开并且异常到达引擎（在那里记录并丢弃它）。</p>
<p>如果它返回一个不可数 <a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_output()</span></code></a> 管道启动，从下一个蜘蛛中间件开始，没有其他 <a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_exception()</span></code></a> 将被调用。</p>
<dl class="field-list simple">
<dt class="field-odd">参数</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>response</strong> (<a class="reference internal" href="request-response.html#scrapy.http.Response" title="scrapy.http.Response"><code class="xref py py-class docutils literal notranslate"><span class="pre">Response</span></code></a> object) -- 引发异常时正在处理的响应</p></li>
<li><p><strong>exception</strong> (<a class="reference external" href="https://docs.python.org/3/library/exceptions.html#Exception" title="(在 Python v3.9)"><code class="xref py py-exc docutils literal notranslate"><span class="pre">Exception</span></code></a> object) -- 引发的异常</p></li>
<li><p><strong>spider</strong> (<a class="reference internal" href="spiders.html#scrapy.spiders.Spider" title="scrapy.spiders.Spider"><code class="xref py py-class docutils literal notranslate"><span class="pre">Spider</span></code></a> object) -- 引发异常的蜘蛛</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="py method">
<dt id="scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests">
<code class="sig-name descname">process_start_requests</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">start_requests</span></em>, <em class="sig-param"><span class="n">spider</span></em><span class="sig-paren">)</span><a class="headerlink" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests" title="永久链接至目标">¶</a></dt>
<dd><p>这个方法是用spider的start请求调用的，其工作原理与 <a class="reference internal" href="#scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output" title="scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output"><code class="xref py py-meth docutils literal notranslate"><span class="pre">process_spider_output()</span></code></a> 方法，但它没有关联的响应，必须只返回请求（而不是项）。</p>
<p>它收到一个不可更改的（在 <code class="docutils literal notranslate"><span class="pre">start_requests</span></code> 参数），并且必须返回另一个ITerable of <a class="reference internal" href="request-response.html#scrapy.http.Request" title="scrapy.http.Request"><code class="xref py py-class docutils literal notranslate"><span class="pre">Request</span></code></a> 物体。</p>
<div class="admonition note">
<p class="admonition-title">注解</p>
<p>在您的蜘蛛中间件中实现此方法时，您应该始终返回一个ITerable（跟随输入的ITerable），而不是全部消费。 <code class="docutils literal notranslate"><span class="pre">start_requests</span></code> 迭代器，因为它可能非常大（甚至没有边界），并导致内存溢出。scrappy引擎设计用于在有能力处理启动请求的情况下提取它们，因此，如果存在停止spider的其他条件（如时间限制或项目/页面计数），则启动请求迭代器实际上可以是无止境的。</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">参数</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>start_requests</strong> (an iterable of <a class="reference internal" href="request-response.html#scrapy.http.Request" title="scrapy.http.Request"><code class="xref py py-class docutils literal notranslate"><span class="pre">Request</span></code></a>) -- 启动请求</p></li>
<li><p><strong>spider</strong> (<a class="reference internal" href="spiders.html#scrapy.spiders.Spider" title="scrapy.spiders.Spider"><code class="xref py py-class docutils literal notranslate"><span class="pre">Spider</span></code></a> object) -- 启动请求所属的蜘蛛</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="py method">
<dt id="scrapy.spidermiddlewares.SpiderMiddleware.from_crawler">
<code class="sig-name descname">from_crawler</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">cls</span></em>, <em class="sig-param"><span class="n">crawler</span></em><span class="sig-paren">)</span><a class="headerlink" href="#scrapy.spidermiddlewares.SpiderMiddleware.from_crawler" title="永久链接至目标">¶</a></dt>
<dd><p>如果存在，则调用该类方法从 <a class="reference internal" href="api.html#scrapy.crawler.Crawler" title="scrapy.crawler.Crawler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Crawler</span></code></a> . 它必须返回中间件的新实例。爬虫对象提供对所有零碎核心组件（如设置和信号）的访问；它是中间件访问它们并将其功能连接到零碎的一种方式。</p>
<dl class="field-list simple">
<dt class="field-odd">参数</dt>
<dd class="field-odd"><p><strong>crawler</strong> (<a class="reference internal" href="api.html#scrapy.crawler.Crawler" title="scrapy.crawler.Crawler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Crawler</span></code></a> object) -- 使用此中间件的爬虫程序</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="built-in-spider-middleware-reference">
<span id="topics-spider-middleware-ref"></span><h2>内置蜘蛛中间件参考<a class="headerlink" href="#built-in-spider-middleware-reference" title="永久链接至标题">¶</a></h2>
<p>本页描述了Scrapy附带的所有蜘蛛中间件组件。有关如何使用它们以及如何编写自己的蜘蛛中间件的信息，请参阅 <a class="reference internal" href="#topics-spider-middleware"><span class="std std-ref">spider middleware usage guide</span></a> .</p>
<p>有关默认启用的组件列表（及其顺序），请参见 <a class="reference internal" href="settings.html#std-setting-SPIDER_MIDDLEWARES_BASE"><code class="xref std std-setting docutils literal notranslate"><span class="pre">SPIDER_MIDDLEWARES_BASE</span></code></a> 设置。</p>
<div class="section" id="module-scrapy.spidermiddlewares.depth">
<span id="depthmiddleware"></span><h3>DepthMiddleware<a class="headerlink" href="#module-scrapy.spidermiddlewares.depth" title="永久链接至标题">¶</a></h3>
<dl class="py class">
<dt id="scrapy.spidermiddlewares.depth.DepthMiddleware">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.depth.</code><code class="sig-name descname">DepthMiddleware</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/depth.html#DepthMiddleware"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.depth.DepthMiddleware" title="永久链接至目标">¶</a></dt>
<dd><p>DepthmIddleware用于跟踪被擦除站点内每个请求的深度。它通过设置工作 <code class="docutils literal notranslate"><span class="pre">request.meta['depth']</span> <span class="pre">=</span> <span class="pre">0</span></code> 如果之前没有设置值（通常只是第一个请求），则将其递增1。</p>
<p>它可以用来限制要抓取的最大深度，根据深度控制请求优先级，等等。</p>
<p>这个 <a class="reference internal" href="#scrapy.spidermiddlewares.depth.DepthMiddleware" title="scrapy.spidermiddlewares.depth.DepthMiddleware"><code class="xref py py-class docutils literal notranslate"><span class="pre">DepthMiddleware</span></code></a> 可以通过以下设置进行配置（有关详细信息，请参阅设置文档）：</p>
<blockquote>
<div><ul class="simple">
<li><p><a class="reference internal" href="settings.html#std-setting-DEPTH_LIMIT"><code class="xref std std-setting docutils literal notranslate"><span class="pre">DEPTH_LIMIT</span></code></a> -允许对任何网站进行爬网的最大深度。如果为零，则不施加限制。</p></li>
<li><p><a class="reference internal" href="settings.html#std-setting-DEPTH_STATS_VERBOSE"><code class="xref std std-setting docutils literal notranslate"><span class="pre">DEPTH_STATS_VERBOSE</span></code></a> -是否收集每个深度的请求数。</p></li>
<li><p><a class="reference internal" href="settings.html#std-setting-DEPTH_PRIORITY"><code class="xref std std-setting docutils literal notranslate"><span class="pre">DEPTH_PRIORITY</span></code></a> -是否根据请求的深度对其进行优先级排序。</p></li>
</ul>
</div></blockquote>
</dd></dl>

</div>
<div class="section" id="module-scrapy.spidermiddlewares.httperror">
<span id="httperrormiddleware"></span><h3>HttpErrorMiddleware<a class="headerlink" href="#module-scrapy.spidermiddlewares.httperror" title="永久链接至标题">¶</a></h3>
<dl class="py class">
<dt id="scrapy.spidermiddlewares.httperror.HttpErrorMiddleware">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.httperror.</code><code class="sig-name descname">HttpErrorMiddleware</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/httperror.html#HttpErrorMiddleware"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.httperror.HttpErrorMiddleware" title="永久链接至目标">¶</a></dt>
<dd><p>过滤掉不成功的（错误的）HTTP响应，这样spider就不必处理它们了，而这些（大部分时间）会增加开销，消耗更多的资源，并使spider逻辑更加复杂。</p>
</dd></dl>

<p>根据 <a class="reference external" href="https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html">HTTP standard</a> ，成功的响应是状态代码在200-300范围内的响应。</p>
<p>如果仍要处理超出该范围的响应代码，可以使用 <code class="docutils literal notranslate"><span class="pre">handle_httpstatus_list</span></code> 蜘蛛属性或 <a class="reference internal" href="#std-setting-HTTPERROR_ALLOWED_CODES"><code class="xref std std-setting docutils literal notranslate"><span class="pre">HTTPERROR_ALLOWED_CODES</span></code></a> 设置。</p>
<p>例如，如果您希望您的蜘蛛处理404响应，可以这样做：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">MySpider</span><span class="p">(</span><span class="n">CrawlSpider</span><span class="p">):</span>
    <span class="n">handle_httpstatus_list</span> <span class="o">=</span> <span class="p">[</span><span class="mi">404</span><span class="p">]</span>
</pre></div>
</div>
<span class="target" id="std-reqmeta-handle_httpstatus_list"><span id="std:reqmeta-handle_httpstatus_list"></span></span><p id="std-reqmeta-handle_httpstatus_all"><span id="std:reqmeta-handle_httpstatus_all"></span>这个 <code class="docutils literal notranslate"><span class="pre">handle_httpstatus_list</span></code> 关键 <a class="reference internal" href="request-response.html#scrapy.http.Request.meta" title="scrapy.http.Request.meta"><code class="xref py py-attr docutils literal notranslate"><span class="pre">Request.meta</span></code></a> 还可以用于指定每个请求允许哪些响应代码。您也可以设置meta键 <code class="docutils literal notranslate"><span class="pre">handle_httpstatus_all</span></code> 到 <code class="docutils literal notranslate"><span class="pre">True</span></code> 如果您想允许请求的任何响应代码。</p>
<p>不过，请记住，处理非200个响应通常是个坏主意，除非你真的知道自己在做什么。</p>
<p><a class="reference external" href="https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html">HTTP Status Code Definitions</a> .</p>
<div class="section" id="httperrormiddleware-settings">
<h4>httpErrorMiddleware设置<a class="headerlink" href="#httperrormiddleware-settings" title="永久链接至标题">¶</a></h4>
<div class="section" id="httperror-allowed-codes">
<span id="std-setting-HTTPERROR_ALLOWED_CODES"></span><span id="std:setting-HTTPERROR_ALLOWED_CODES"></span><h5>HTTPERROR_ALLOWED_CODES<a class="headerlink" href="#httperror-allowed-codes" title="永久链接至标题">¶</a></h5>
<p>违约： <code class="docutils literal notranslate"><span class="pre">[]</span></code></p>
<p>通过此列表中包含非200状态代码的所有响应。</p>
</div>
<div class="section" id="httperror-allow-all">
<span id="std-setting-HTTPERROR_ALLOW_ALL"></span><span id="std:setting-HTTPERROR_ALLOW_ALL"></span><h5>HTTPERROR_ALLOW_ALL<a class="headerlink" href="#httperror-allow-all" title="永久链接至标题">¶</a></h5>
<p>违约： <code class="docutils literal notranslate"><span class="pre">False</span></code></p>
<p>传递所有响应，无论其状态代码如何。</p>
</div>
</div>
</div>
<div class="section" id="module-scrapy.spidermiddlewares.offsite">
<span id="offsitemiddleware"></span><h3>OffsiteMiddleware<a class="headerlink" href="#module-scrapy.spidermiddlewares.offsite" title="永久链接至标题">¶</a></h3>
<dl class="py class">
<dt id="scrapy.spidermiddlewares.offsite.OffsiteMiddleware">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.offsite.</code><code class="sig-name descname">OffsiteMiddleware</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/offsite.html#OffsiteMiddleware"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.offsite.OffsiteMiddleware" title="永久链接至目标">¶</a></dt>
<dd><p>过滤掉对spider所覆盖域之外的URL的请求。</p>
<p>这个中间件过滤掉主机名不在spider中的每个请求 <a class="reference internal" href="spiders.html#scrapy.spiders.Spider.allowed_domains" title="scrapy.spiders.Spider.allowed_domains"><code class="xref py py-attr docutils literal notranslate"><span class="pre">allowed_domains</span></code></a> 属性。也允许列表中任何域的所有子域。例如规则 <code class="docutils literal notranslate"><span class="pre">www.example.org</span></code> 也将允许 <code class="docutils literal notranslate"><span class="pre">bob.www.example.org</span></code> 但不是 <code class="docutils literal notranslate"><span class="pre">www2.example.com</span></code> 也不 <code class="docutils literal notranslate"><span class="pre">example.com</span></code> .</p>
<p>当您的spider返回一个不属于spider覆盖的域的请求时，这个中间件将记录一个与这个类似的调试消息：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">DEBUG</span><span class="p">:</span> <span class="n">Filtered</span> <span class="n">offsite</span> <span class="n">request</span> <span class="n">to</span> <span class="s1">&#39;www.othersite.com&#39;</span><span class="p">:</span> <span class="o">&lt;</span><span class="n">GET</span> <span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">www</span><span class="o">.</span><span class="n">othersite</span><span class="o">.</span><span class="n">com</span><span class="o">/</span><span class="n">some</span><span class="o">/</span><span class="n">page</span><span class="o">.</span><span class="n">html</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>为了避免在日志中填充过多的噪声，它将只为每个过滤后的新域打印其中一条消息。例如，如果另一个请求 <code class="docutils literal notranslate"><span class="pre">www.othersite.com</span></code> 已筛选，将不打印日志消息。但如果有人要求 <code class="docutils literal notranslate"><span class="pre">someothersite.com</span></code> 过滤后，将打印一条消息（但仅针对过滤后的第一个请求）。</p>
<p>如果蜘蛛没有定义 <a class="reference internal" href="spiders.html#scrapy.spiders.Spider.allowed_domains" title="scrapy.spiders.Spider.allowed_domains"><code class="xref py py-attr docutils literal notranslate"><span class="pre">allowed_domains</span></code></a> 属性，或者属性为空，异地中间件将允许所有请求。</p>
<p>如果请求具有 <code class="xref py py-attr docutils literal notranslate"><span class="pre">dont_filter</span></code> 属性集，非现场中间件将允许请求，即使其域未在允许的域中列出。</p>
</dd></dl>

</div>
<div class="section" id="module-scrapy.spidermiddlewares.referer">
<span id="referermiddleware"></span><h3>RefererMiddleware<a class="headerlink" href="#module-scrapy.spidermiddlewares.referer" title="永久链接至标题">¶</a></h3>
<dl class="py class">
<dt id="scrapy.spidermiddlewares.referer.RefererMiddleware">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.referer.</code><code class="sig-name descname">RefererMiddleware</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/referer.html#RefererMiddleware"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.referer.RefererMiddleware" title="永久链接至目标">¶</a></dt>
<dd><p>填充请求 <code class="docutils literal notranslate"><span class="pre">Referer</span></code> 头，基于生成它的响应的URL。</p>
</dd></dl>

<div class="section" id="referermiddleware-settings">
<h4>ReferermIddleware设置<a class="headerlink" href="#referermiddleware-settings" title="永久链接至标题">¶</a></h4>
<div class="section" id="referer-enabled">
<span id="std-setting-REFERER_ENABLED"></span><span id="std:setting-REFERER_ENABLED"></span><h5>REFERER_ENABLED<a class="headerlink" href="#referer-enabled" title="永久链接至标题">¶</a></h5>
<p>违约： <code class="docutils literal notranslate"><span class="pre">True</span></code></p>
<p>是否启用引用中间件。</p>
</div>
<div class="section" id="referrer-policy">
<span id="std-setting-REFERRER_POLICY"></span><span id="std:setting-REFERRER_POLICY"></span><h5>REFERRER_POLICY<a class="headerlink" href="#referrer-policy" title="永久链接至标题">¶</a></h5>
<p>违约： <code class="docutils literal notranslate"><span class="pre">'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'</span></code></p>
<p id="std-reqmeta-referrer_policy"><span id="std:reqmeta-referrer_policy"></span><a class="reference external" href="https://www.w3.org/TR/referrer-policy">Referrer Policy</a> 填充请求“引用”头时应用。</p>
<div class="admonition note">
<p class="admonition-title">注解</p>
<p>您还可以使用特殊的 <code class="docutils literal notranslate"><span class="pre">&quot;referrer_policy&quot;</span></code>  <a class="reference internal" href="request-response.html#topics-request-meta"><span class="std std-ref">Request.meta</span></a> 键，其可接受值与 <code class="docutils literal notranslate"><span class="pre">REFERRER_POLICY</span></code> 设置。</p>
</div>
<div class="section" id="acceptable-values-for-referrer-policy">
<h6>推荐人政策的可接受值<a class="headerlink" href="#acceptable-values-for-referrer-policy" title="永久链接至标题">¶</a></h6>
<ul class="simple">
<li><p>或者是到 <code class="docutils literal notranslate"><span class="pre">scrapy.spidermiddlewares.referer.ReferrerPolicy</span></code> 子类-自定义策略或内置策略之一（参见下面的类）。</p></li>
<li><p>或标准W3C定义的字符串值之一，</p></li>
<li><p>或特殊 <code class="docutils literal notranslate"><span class="pre">&quot;scrapy-default&quot;</span></code> .</p></li>
</ul>
<table class="docutils align-default">
<colgroup>
<col style="width: 34%" />
<col style="width: 66%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"><p>字符串值</p></th>
<th class="head"><p>类名（作为字符串）</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">&quot;scrapy-default&quot;</span></code> （默认）</p></td>
<td><p><a class="reference internal" href="#scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" title="scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"><code class="xref py py-class docutils literal notranslate"><span class="pre">scrapy.spidermiddlewares.referer.DefaultReferrerPolicy</span></code></a></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference external" href="https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer">&quot;no-referrer&quot;</a></p></td>
<td><p><a class="reference internal" href="#scrapy.spidermiddlewares.referer.NoReferrerPolicy" title="scrapy.spidermiddlewares.referer.NoReferrerPolicy"><code class="xref py py-class docutils literal notranslate"><span class="pre">scrapy.spidermiddlewares.referer.NoReferrerPolicy</span></code></a></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference external" href="https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade">&quot;no-referrer-when-downgrade&quot;</a></p></td>
<td><p><a class="reference internal" href="#scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy" title="scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"><code class="xref py py-class docutils literal notranslate"><span class="pre">scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy</span></code></a></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference external" href="https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin">&quot;same-origin&quot;</a></p></td>
<td><p><a class="reference internal" href="#scrapy.spidermiddlewares.referer.SameOriginPolicy" title="scrapy.spidermiddlewares.referer.SameOriginPolicy"><code class="xref py py-class docutils literal notranslate"><span class="pre">scrapy.spidermiddlewares.referer.SameOriginPolicy</span></code></a></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference external" href="https://www.w3.org/TR/referrer-policy/#referrer-policy-origin">&quot;origin&quot;</a></p></td>
<td><p><a class="reference internal" href="#scrapy.spidermiddlewares.referer.OriginPolicy" title="scrapy.spidermiddlewares.referer.OriginPolicy"><code class="xref py py-class docutils literal notranslate"><span class="pre">scrapy.spidermiddlewares.referer.OriginPolicy</span></code></a></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference external" href="https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin">&quot;strict-origin&quot;</a></p></td>
<td><p><a class="reference internal" href="#scrapy.spidermiddlewares.referer.StrictOriginPolicy" title="scrapy.spidermiddlewares.referer.StrictOriginPolicy"><code class="xref py py-class docutils literal notranslate"><span class="pre">scrapy.spidermiddlewares.referer.StrictOriginPolicy</span></code></a></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference external" href="https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin">&quot;origin-when-cross-origin&quot;</a></p></td>
<td><p><a class="reference internal" href="#scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy" title="scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"><code class="xref py py-class docutils literal notranslate"><span class="pre">scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy</span></code></a></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference external" href="https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin">&quot;strict-origin-when-cross-origin&quot;</a></p></td>
<td><p><a class="reference internal" href="#scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy" title="scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy"><code class="xref py py-class docutils literal notranslate"><span class="pre">scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy</span></code></a></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference external" href="https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url">&quot;unsafe-url&quot;</a></p></td>
<td><p><a class="reference internal" href="#scrapy.spidermiddlewares.referer.UnsafeUrlPolicy" title="scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"><code class="xref py py-class docutils literal notranslate"><span class="pre">scrapy.spidermiddlewares.referer.UnsafeUrlPolicy</span></code></a></p></td>
</tr>
</tbody>
</table>
<dl class="py class">
<dt id="scrapy.spidermiddlewares.referer.DefaultReferrerPolicy">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.referer.</code><code class="sig-name descname">DefaultReferrerPolicy</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/referer.html#DefaultReferrerPolicy"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" title="永久链接至目标">¶</a></dt>
<dd><p>“降级时无引用”的变体，加上如果父请求正在使用，则不发送“引用”。 <code class="docutils literal notranslate"><span class="pre">file://</span></code> 或 <code class="docutils literal notranslate"><span class="pre">s3://</span></code> 方案。</p>
</dd></dl>

<div class="admonition warning">
<p class="admonition-title">警告</p>
<p>Scrapy的默认引用策略-就像 <a class="reference external" href="https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade">&quot;no-referrer-when-downgrade&quot;</a> ，W3C推荐的浏览器值-将从任何 <code class="docutils literal notranslate"><span class="pre">http(s)://</span></code> 对任何 <code class="docutils literal notranslate"><span class="pre">https://</span></code> URL，即使域不同。</p>
<p><a class="reference external" href="https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin">&quot;same-origin&quot;</a> 如果要删除跨域请求的引用者信息，可能是更好的选择。</p>
</div>
<dl class="py class">
<dt id="scrapy.spidermiddlewares.referer.NoReferrerPolicy">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.referer.</code><code class="sig-name descname">NoReferrerPolicy</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/referer.html#NoReferrerPolicy"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.referer.NoReferrerPolicy" title="永久链接至目标">¶</a></dt>
<dd><p><a class="reference external" href="https://www.w3.org/tr/referer-policy/referer-policy-no-referer">https://www.w3.org/tr/referer-policy/referer-policy-no-referer</a></p>
<p>最简单的策略是“无引用”，它指定不将引用信息与从特定请求客户机发出的请求一起发送到任何源站。标题将完全省略。</p>
</dd></dl>

<dl class="py class">
<dt id="scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.referer.</code><code class="sig-name descname">NoReferrerWhenDowngradePolicy</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/referer.html#NoReferrerWhenDowngradePolicy"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy" title="永久链接至目标">¶</a></dt>
<dd><p><a class="reference external" href="https://www.w3.org/tr/referer-policy">https://www.w3.org/tr/referer-policy</a>/降级时没有referer-policy</p>
<p>“降级时不引用”策略将完整的URL以及来自受TLS保护的环境设置对象的请求发送到可能值得信任的URL，以及来自不受TLS保护的客户端的请求发送到任何源站。</p>
<p>另一方面，来自受TLS保护的客户端对不可能可信的URL的请求将不包含引用者信息。将不发送引用HTTP头。</p>
<p>如果没有另外指定策略，这是用户代理的默认行为。</p>
</dd></dl>

<div class="admonition note">
<p class="admonition-title">注解</p>
<p>“降级时不引用”策略是W3C推荐的默认策略，主要Web浏览器使用该策略。</p>
<p>但是，它不是Scrapy的默认引用者策略（请参见 <a class="reference internal" href="#scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" title="scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"><code class="xref py py-class docutils literal notranslate"><span class="pre">DefaultReferrerPolicy</span></code></a> ）</p>
</div>
<dl class="py class">
<dt id="scrapy.spidermiddlewares.referer.SameOriginPolicy">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.referer.</code><code class="sig-name descname">SameOriginPolicy</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/referer.html#SameOriginPolicy"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.referer.SameOriginPolicy" title="永久链接至目标">¶</a></dt>
<dd><p><a class="reference external" href="https://www.w3.org/tr/referer-policy/referer-policy-same-origin">https://www.w3.org/tr/referer-policy/referer-policy-same-origin</a></p>
<p>“同一来源”策略指定，当从特定请求客户端发出同一来源请求时，删除作为引用的完整URL作为引用信息发送。</p>
<p>另一方面，跨源请求将不包含引用者信息。将不发送引用HTTP头。</p>
</dd></dl>

<dl class="py class">
<dt id="scrapy.spidermiddlewares.referer.OriginPolicy">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.referer.</code><code class="sig-name descname">OriginPolicy</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/referer.html#OriginPolicy"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.referer.OriginPolicy" title="永久链接至目标">¶</a></dt>
<dd><p><a class="reference external" href="https://www.w3.org/tr/referer-policy/referer-policy-origin">https://www.w3.org/tr/referer-policy/referer-policy-origin</a></p>
<p>“来源”策略指定，在从特定请求客户机发出相同的来源请求和跨来源请求时，仅将请求客户机来源的ASCII序列化作为引用信息发送。</p>
</dd></dl>

<dl class="py class">
<dt id="scrapy.spidermiddlewares.referer.StrictOriginPolicy">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.referer.</code><code class="sig-name descname">StrictOriginPolicy</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/referer.html#StrictOriginPolicy"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.referer.StrictOriginPolicy" title="永久链接至目标">¶</a></dt>
<dd><p><a class="reference external" href="https://www.w3.org/tr/referer-policy/referer-policy-strict-origin">https://www.w3.org/tr/referer-policy/referer-policy-strict-origin</a></p>
<p>“严格来源”策略在发出请求时发送请求客户端来源的ASCII序列化：从受TLS保护的环境设置对象发送到可能值得信任的URL，从不受TLS保护的环境设置对象发送到任何来源。</p>
<p>另一方面，来自受TLS保护的请求客户端对不可能可信的URL的请求将不包含引用者信息。将不发送引用HTTP头。</p>
</dd></dl>

<dl class="py class">
<dt id="scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.referer.</code><code class="sig-name descname">OriginWhenCrossOriginPolicy</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/referer.html#OriginWhenCrossOriginPolicy"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy" title="永久链接至目标">¶</a></dt>
<dd><p><a class="reference external" href="https://www.w3.org/tr/referer-policy">https://www.w3.org/tr/referer-policy</a>/跨来源引用策略来源</p>
<p>“跨源站时的源站”策略指定，当从特定请求客户机发出相同的源站请求时，剥离用作引用的完整URL作为引用信息发送，而在从特定请求客户机发出跨源站请求时，仅将请求客户机源站的ASCII序列化作为引用信息发送。请求客户端。</p>
</dd></dl>

<dl class="py class">
<dt id="scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.referer.</code><code class="sig-name descname">StrictOriginWhenCrossOriginPolicy</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/referer.html#StrictOriginWhenCrossOriginPolicy"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy" title="永久链接至目标">¶</a></dt>
<dd><p><a class="reference external" href="https://www.w3.org/tr/referer-policy">https://www.w3.org/tr/referer-policy</a>/跨来源时引用策略严格来源</p>
<p>“跨来源时严格来源”策略指定，当从特定请求客户端发出同一来源请求时，剥离用作引用的完整URL作为引用信息发送，并且在发出跨来源请求时，仅将请求客户端来源的ASCII序列化作为引用信息发送：</p>
<ul class="simple">
<li><p>从受TLS保护的环境设置对象到可能值得信赖的URL，以及</p></li>
<li><p>从非TLS保护的环境设置对象到任何源站。</p></li>
</ul>
<p>另一方面，来自受TLS保护的客户端对不可能可信的URL的请求将不包含引用者信息。将不发送引用HTTP头。</p>
</dd></dl>

<dl class="py class">
<dt id="scrapy.spidermiddlewares.referer.UnsafeUrlPolicy">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.referer.</code><code class="sig-name descname">UnsafeUrlPolicy</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/referer.html#UnsafeUrlPolicy"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.referer.UnsafeUrlPolicy" title="永久链接至目标">¶</a></dt>
<dd><p><a class="reference external" href="https://www.w3.org/tr/referer-policy/referer-policy-unsafe-url">https://www.w3.org/tr/referer-policy/referer-policy-unsafe-url</a></p>
<p>“不安全的URL”策略指定一个完整的URL，被剥离用作引用，与来自特定请求客户端的跨源请求和同一源请求一起发送。</p>
<p>注意：保险单的名称不是谎言，它是不安全的。此策略将泄漏从受TLS保护的资源到不安全源的源和路径。仔细考虑为可能敏感的文档设置此类策略的影响。</p>
</dd></dl>

<div class="admonition warning">
<p class="admonition-title">警告</p>
<p>不建议使用“不安全URL”策略。</p>
</div>
</div>
</div>
</div>
</div>
<div class="section" id="module-scrapy.spidermiddlewares.urllength">
<span id="urllengthmiddleware"></span><h3>UrlLengthMiddleware<a class="headerlink" href="#module-scrapy.spidermiddlewares.urllength" title="永久链接至标题">¶</a></h3>
<dl class="py class">
<dt id="scrapy.spidermiddlewares.urllength.UrlLengthMiddleware">
<em class="property">class </em><code class="sig-prename descclassname">scrapy.spidermiddlewares.urllength.</code><code class="sig-name descname">UrlLengthMiddleware</code><a class="reference internal" href="../_modules/scrapy/spidermiddlewares/urllength.html#UrlLengthMiddleware"><span class="viewcode-link">[源代码]</span></a><a class="headerlink" href="#scrapy.spidermiddlewares.urllength.UrlLengthMiddleware" title="永久链接至目标">¶</a></dt>
<dd><p>筛选出URL超过URL长度限制的请求</p>
<p>这个 <a class="reference internal" href="#scrapy.spidermiddlewares.urllength.UrlLengthMiddleware" title="scrapy.spidermiddlewares.urllength.UrlLengthMiddleware"><code class="xref py py-class docutils literal notranslate"><span class="pre">UrlLengthMiddleware</span></code></a> 可以通过以下设置进行配置（有关详细信息，请参阅设置文档）：</p>
<blockquote>
<div><ul class="simple">
<li><p><a class="reference internal" href="settings.html#std-setting-URLLENGTH_LIMIT"><code class="xref std std-setting docutils literal notranslate"><span class="pre">URLLENGTH_LIMIT</span></code></a> -允许已爬网URL的最大URL长度。</p></li>
</ul>
</div></blockquote>
</dd></dl>

</div>
</div>
</div>


           </div>
           
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="extensions.html" class="btn btn-neutral float-right" title="扩展" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="downloader-middleware.html" class="btn btn-neutral float-left" title="下载器中间件" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        
        &copy; 版权所有 2008–2020, Scrapy developers
      <span class="lastupdated">
        最后更新于 10月 18, 2020.
      </span>

    </p>
  </div>
    
    
    
    Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a
    
    <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a>
    
    provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
  
 
<script type="text/javascript">
!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t){var e=document.createElement("script");e.type="text/javascript";e.async=!0;e.src=("https:"===document.location.protocol?"https://":"http://")+"cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(e,n)};analytics.SNIPPET_VERSION="3.1.0";
analytics.load("8UDQfnf3cyFSTsM4YANnW5sXmgZVILbA");
analytics.page();
}}();

analytics.ready(function () {
    ga('require', 'linker');
    ga('linker:autoLink', ['scrapinghub.com', 'crawlera.com']);
});
</script>


</body>
</html>